#!/usr/bin/env python
# -*- coding: utf-8 -*-
import hashlib
import json
import os
import re


class UrlManager(object):

    def __init__(self):
        self.rootPath = "./urls/"
        self.newPath = "new"
        self.handingUrl = 'handingUrl'
        self.newUrls = set()
        self.oldUrls = set()
        self.errorUrls = set()

    def addNewUrl(self, url):
        newFilePath = self._getNewFilePath(url)
        oldFilePath = self._getOldFilePath(url)
        errFilePath = self._getErrFilePath(url)

        # 判断文件是否存在
        if not os.path.exists(newFilePath) and not os.path.exists(oldFilePath) and not os.path.exists(errFilePath):
            print("==== 新增URL：%s ====" % url)
            self._saveFileUrl(filePath = newFilePath, url = url)

    # 判断是否有带爬取URL
    def hasNewUrl(self):
        dirPath = os.path.join(self.rootPath, self.newPath)
        return bool(os.listdir(dirPath))

    def getNewUrl(self):
        filePath = _fileName = None
        # 获取新url，并将其在newUrls中删除
        dirPath = os.path.join(self.rootPath, self.newPath)
        newUrls = os.listdir(dirPath)
        # 获取当前处理的url
        handingUrlPath = os.path.join(self.rootPath, self.handingUrl)

        if os.path.exists(handingUrlPath) :
            handingUrlContent = open(handingUrlPath, encoding='utf-8', mode='r').read().strip()
            handingUrlContent = handingUrlContent.split('_')

            if bool(handingUrlContent) and len(handingUrlContent) == 2 :
                for file in newUrls:
                    filePath = os.path.join(dirPath, file)
                    eqFile = file.split('_')
                    if len(eqFile) == 2 and eqFile[0] == handingUrlContent[0]:
                        _fileName = file
                        break

        if not bool(filePath) or not bool(_fileName):
            for file in newUrls:
                _fileName = file
                filePath = os.path.join(dirPath, file)
                break

        # 获取路径
        fileContent = open(filePath, encoding='utf-8', mode='r').read().strip()
        fileContent = json.loads(fileContent)
        newUrl = fileContent['url']
        # 删除
        os.unlink(filePath)
        # 新增
        oldFilePath = self._getOldFilePath(url=newUrl)
        self._saveFileUrl(filePath=oldFilePath, url=newUrl)
        # 保存正在处理的url
        self._saveFileStr(filePath = handingUrlPath, str = _fileName)
        # 返回取出的url
        return newUrl

    # 批量添加URL
    def addNewUrls(self, urls):
        if bool(urls):
            for url in urls:
                self.addNewUrl(url)

    def addErrorUrl(self, url):
        # 将url加入errorUrls,并将其在其他集合中移除
        newFilePath = self._getNewFilePath(url)
        oldFilePath = self._getOldFilePath(url)
        errFilePath = self._getErrFilePath(url)
        if not os.path.exists(errFilePath):
            self._saveFileUrl(filePath=errFilePath, url=url)

        if os.path.exists(newFilePath):
            os.unlink(newFilePath)

        if os.path.exists(oldFilePath):
            os.unlink(oldFilePath)

    # 处理文件路径
    def _getUrlFilePath(self, url, fileName):
        compile = re.compile(r"(\d+)(\_\d+)?\.html$")
        urlSplit = re.findall(compile, url)
        if  len(urlSplit) :
            urlNum, sortNum = urlSplit[0]
            sortNum = sortNum.strip('').lstrip('_')
            sortNum = 0 if not bool(sortNum) else int(sortNum)
            # flag
            tempUrl = re.sub(compile, urlNum + ".html", url)
            return os.path.join(self.rootPath, fileName, hashlib.md5(tempUrl.encode("utf-8")).hexdigest() + '_%d' % sortNum)
        else:
            return os.path.join(self.rootPath, fileName, hashlib.md5(url.encode("utf-8")).hexdigest())

    def _getNewFilePath(self, url):
        return self._getUrlFilePath(url, self.newPath)

    def _getOldFilePath(self, url):
        return self._getUrlFilePath(url, 'old')

    def _getErrFilePath(self, url):
        return self._getUrlFilePath(url, 'error')

    # 保存数据
    def _saveFileUrl(self, filePath, url):
        self._mkdirPath(filePath)
        file = open(filePath, "w")
        file.write(json.dumps({"url": url}))
        file.close()

    # 创建文件夹
    def _mkdirPath(self, filePath):
        dirname = os.path.dirname(filePath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

    def _saveFileStr(self, filePath, str):
        self._mkdirPath(filePath)
        file = open(filePath, "w")
        file.write(str)
        file.close()
